import matplotlib.pyplot as plt
import numpy as np
import statsmodels.api as sm
import statsmodels.stats.diagnostic as smd
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error,r2_score
from sklearn.linear_model import Ridge
import seaborn as sns
import statsmodels.api as sm
from statsmodels.graphics.gofplots import qqplot
from yellowbrick.regressor import ResidualsPlot
import scipy.stats as stats
import statsmodels.stats.diagnostic as diag
from scipy.stats import normaltest
from scipy.stats import linregress
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import NearestNeighbors
from sklearn.cluster import KMeans
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster
from sklearn.decomposition import PCA
from sklearn.metrics import *
import sklearn.metrics as metrics
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules
from sklearn.preprocessing import KBinsDiscretizer
from sklearn.feature_selection import SequentialFeatureSelector 
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_validate
from sklearn.linear_model import RidgeCV, Ridge
from sklearn.linear_model import LassoCV,Lasso

archivo1_csv = "train.csv"
datost = pd.read_csv(archivo1_csv, encoding='ISO-8859-1')

datost.describe()

datost.head()

datost.columns

Index(['Id', 'MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street',
       'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig',
       'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType',
       'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd',
       'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType',
       'MasVnrArea', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual',
       'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1',
       'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating',
       'HeatingQC', 'CentralAir', 'Electrical', '1stFlrSF', '2ndFlrSF',
       'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
       'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual',
       'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'FireplaceQu', 'GarageType',
       'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'GarageQual',
       'GarageCond', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF',
       'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'PoolQC',
       'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType',
       'SaleCondition', 'SalePrice'],
      dtype='object')

datost.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1460 entries, 0 to 1459
Data columns (total 81 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   Id             1460 non-null   int64  
 1   MSSubClass     1460 non-null   int64  
 2   MSZoning       1460 non-null   object 
 3   LotFrontage    1201 non-null   float64
 4   LotArea        1460 non-null   int64  
 5   Street         1460 non-null   object 
 6   Alley          91 non-null     object 
 7   LotShape       1460 non-null   object 
 8   LandContour    1460 non-null   object 
 9   Utilities      1460 non-null   object 
 10  LotConfig      1460 non-null   object 
 11  LandSlope      1460 non-null   object 
 12  Neighborhood   1460 non-null   object 
 13  Condition1     1460 non-null   object 
 14  Condition2     1460 non-null   object 
 15  BldgType       1460 non-null   object 
 16  HouseStyle     1460 non-null   object 
 17  OverallQual    1460 non-null   int64  
 18  OverallCond    1460 non-null   int64  
 19  YearBuilt      1460 non-null   int64  
 20  YearRemodAdd   1460 non-null   int64  
 21  RoofStyle      1460 non-null   object 
 22  RoofMatl       1460 non-null   object 
 23  Exterior1st    1460 non-null   object 
 24  Exterior2nd    1460 non-null   object 
 25  MasVnrType     1452 non-null   object 
 26  MasVnrArea     1452 non-null   float64
 27  ExterQual      1460 non-null   object 
 28  ExterCond      1460 non-null   object 
 29  Foundation     1460 non-null   object 
 30  BsmtQual       1423 non-null   object 
 31  BsmtCond       1423 non-null   object 
 32  BsmtExposure   1422 non-null   object 
 33  BsmtFinType1   1423 non-null   object 
 34  BsmtFinSF1     1460 non-null   int64  
 35  BsmtFinType2   1422 non-null   object 
 36  BsmtFinSF2     1460 non-null   int64  
 37  BsmtUnfSF      1460 non-null   int64  
 38  TotalBsmtSF    1460 non-null   int64  
 39  Heating        1460 non-null   object 
 40  HeatingQC      1460 non-null   object 
 41  CentralAir     1460 non-null   object 
 42  Electrical     1459 non-null   object 
 43  1stFlrSF       1460 non-null   int64  
 44  2ndFlrSF       1460 non-null   int64  
 45  LowQualFinSF   1460 non-null   int64  
 46  GrLivArea      1460 non-null   int64  
 47  BsmtFullBath   1460 non-null   int64  
 48  BsmtHalfBath   1460 non-null   int64  
 49  FullBath       1460 non-null   int64  
 50  HalfBath       1460 non-null   int64  
 51  BedroomAbvGr   1460 non-null   int64  
 52  KitchenAbvGr   1460 non-null   int64  
 53  KitchenQual    1460 non-null   object 
 54  TotRmsAbvGrd   1460 non-null   int64  
 55  Functional     1460 non-null   object 
 56  Fireplaces     1460 non-null   int64  
 57  FireplaceQu    770 non-null    object 
 58  GarageType     1379 non-null   object 
 59  GarageYrBlt    1379 non-null   float64
 60  GarageFinish   1379 non-null   object 
 61  GarageCars     1460 non-null   int64  
 62  GarageArea     1460 non-null   int64  
 63  GarageQual     1379 non-null   object 
 64  GarageCond     1379 non-null   object 
 65  PavedDrive     1460 non-null   object 
 66  WoodDeckSF     1460 non-null   int64  
 67  OpenPorchSF    1460 non-null   int64  
 68  EnclosedPorch  1460 non-null   int64  
 69  3SsnPorch      1460 non-null   int64  
 70  ScreenPorch    1460 non-null   int64  
 71  PoolArea       1460 non-null   int64  
 72  PoolQC         7 non-null      object 
 73  Fence          281 non-null    object 
 74  MiscFeature    54 non-null     object 
 75  MiscVal        1460 non-null   int64  
 76  MoSold         1460 non-null   int64  
 77  YrSold         1460 non-null   int64  
 78  SaleType       1460 non-null   object 
 79  SaleCondition  1460 non-null   object 
 80  SalePrice      1460 non-null   int64  
dtypes: float64(3), int64(35), object(43)
memory usage: 924.0+ KB

datost.isnull().sum()

Id                 0
MSSubClass         0
MSZoning           0
LotFrontage      259
LotArea            0
                ... 
MoSold             0
YrSold             0
SaleType           0
SaleCondition      0
SalePrice          0
Length: 81, dtype: int64

datost.duplicated().sum()

0

# Obtener los nombres de las columnas numericas
columnas_numericas = datost.select_dtypes(include=['number']).columns.tolist()
columnas_numericas.remove('Id')

# Obtener los nombres de las columnas no numéricas
columnas_no_numericas = datost.select_dtypes(exclude=['number']).columns.tolist()
print("Columnas numéricas:")
print(columnas_numericas)

print("\nColumnas no numéricas:")
print(columnas_no_numericas)

numericast = datost[columnas_numericas]
#numericasp = datosp[columnas_numericas]

Columnas numéricas:
['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'MoSold', 'YrSold', 'SalePrice']

Columnas no numéricas:
['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual', 'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']

def count_empty_values_per_column(df):
    total_rows = len(df)
    missing_values = df.isna().sum()
    missing_values = missing_values[missing_values > 0]  # Filtrar solo columnas con valores faltantes
    
    if missing_values.empty:
        print("No hay valores faltantes en el DataFrame.")
    else:
        missing_percentage = (missing_values / total_rows) * 100
        missing_info = pd.DataFrame({
            'Valores Faltantes': missing_values,
            'Proporción (%)': missing_percentage
        })
        print("Número de valores faltantes por columna y su proporción respecto al total de filas:")
        print(missing_info)

#Columnas del df de train con filas vacias
count_empty_values_per_column(numericast)

Número de valores faltantes por columna y su proporción respecto al total de filas:
             Valores Faltantes  Proporción (%)
LotFrontage                259       17.739726
MasVnrArea                   8        0.547945
GarageYrBlt                 81        5.547945

#Columnas del df de test con filas vacias
#count_empty_values_per_column(numericasp)

def plot_scatter_and_hist(df, x_columns, y_column):
    num_plots = len(x_columns)
    fig, axes = plt.subplots(num_plots + 1, 1, figsize=(6, 4 * (num_plots + 1)))
    
    if num_plots == 1:
        axes = [axes]  # Asegurar que axes es iterable
    
    # Generar diagramas de dispersión y calcular R²
    for i, x_col in enumerate(x_columns):
        sns.scatterplot(data=df, x=x_col, y=y_column, ax=axes[i])
        axes[i].set_title(f'Diagrama de dispersión: {x_col} vs {y_column}')
        
        # Calcular R²
        valid_data = df[[x_col, y_column]].dropna()
        if not valid_data.empty:
            slope, intercept, r_value, _, _ = linregress(valid_data[x_col], valid_data[y_column])
            r_squared = r_value ** 2
            axes[i].text(0.05, 0.9, f'R² = {r_squared:.4f}', transform=axes[i].transAxes, fontsize=12, 
                         bbox=dict(facecolor='white', alpha=0.5))
    
    # Generar histograma de y_column
    sns.histplot(df[y_column], bins=20, kde=True, ax=axes[-1])
    axes[-1].set_title(f'Histograma de {y_column}')
    
    plt.tight_layout()
    plt.show()

cand = columnas_numericas
#print(cand)
cand.remove('SalePrice')
plot_scatter_and_hist(numericast, cand, 'SalePrice')

# Seleccionar las variables clave
columnas_numericas = ['SalePrice','MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd'
                      , 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 
                      'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath'
                      , 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt'
                      , 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch'
                      , '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'MoSold', 'YrSold']

datos_numericos = datost[columnas_numericas]  # Crear nuevo dataframe con las variables seleccionadas

# Mostrar las primeras filas para verificar
datos_numericos.head()

print("Valores nulos por columna:")
print(datos_numericos.isnull().sum())

# Verificar valores duplicados
print("\nNúmero de filas duplicadas:")
print(datos_numericos.duplicated().sum())

# Reemplazar valores nulos con la mediana de cada columna
# Para columnas numéricas, sustituimos los valores nulos por la mediana
datos_numericos = datos_numericos.apply(lambda x: x.fillna(x.median()) if x.isnull().any() else x)

# Verificar nuevamente los valores nulos después de reemplazarlos
print("\nValores nulos después de reemplazar con la mediana:")
print(datos_numericos.isnull().sum())

Valores nulos por columna:
SalePrice          0
MSSubClass         0
LotFrontage      259
LotArea            0
OverallQual        0
OverallCond        0
YearBuilt          0
YearRemodAdd       0
MasVnrArea         8
BsmtFinSF1         0
BsmtFinSF2         0
BsmtUnfSF          0
TotalBsmtSF        0
1stFlrSF           0
2ndFlrSF           0
LowQualFinSF       0
GrLivArea          0
BsmtFullBath       0
BsmtHalfBath       0
FullBath           0
HalfBath           0
BedroomAbvGr       0
KitchenAbvGr       0
TotRmsAbvGrd       0
Fireplaces         0
GarageYrBlt       81
GarageCars         0
GarageArea         0
WoodDeckSF         0
OpenPorchSF        0
EnclosedPorch      0
3SsnPorch          0
ScreenPorch        0
PoolArea           0
MiscVal            0
MoSold             0
YrSold             0
dtype: int64

Número de filas duplicadas:
0

Valores nulos después de reemplazar con la mediana:
SalePrice        0
MSSubClass       0
LotFrontage      0
LotArea          0
OverallQual      0
OverallCond      0
YearBuilt        0
YearRemodAdd     0
MasVnrArea       0
BsmtFinSF1       0
BsmtFinSF2       0
BsmtUnfSF        0
TotalBsmtSF      0
1stFlrSF         0
2ndFlrSF         0
LowQualFinSF     0
GrLivArea        0
BsmtFullBath     0
BsmtHalfBath     0
FullBath         0
HalfBath         0
BedroomAbvGr     0
KitchenAbvGr     0
TotRmsAbvGrd     0
Fireplaces       0
GarageYrBlt      0
GarageCars       0
GarageArea       0
WoodDeckSF       0
OpenPorchSF      0
EnclosedPorch    0
3SsnPorch        0
ScreenPorch      0
PoolArea         0
MiscVal          0
MoSold           0
YrSold           0
dtype: int64

X = datos_numericos[columnas_numericas]

# Calcular matriz de correlaciones
corr_matrix = X.corr()


# Ajustar el tamaño de la figura para que sea lo suficientemente grande
plt.figure(figsize=(16, 14))

# Crear el heatmap
sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', center=0, 
            cbar_kws={'shrink': 0.8},  # Reducir el tamaño de la barra de color
            annot_kws={'size': 8},  # Reducir el tamaño de la fuente de los números
            linewidths=0.5)

# Rotar las etiquetas para que sean legibles
plt.xticks(rotation=90)  # Rota las etiquetas del eje X
plt.yticks(rotation=0)   # Rota las etiquetas del eje Y (si es necesario)

# Agregar título
plt.title('Matriz de Correlaciones')

# Ajustar los márgenes para que no se corten las etiquetas
plt.tight_layout()

# Mostrar la matriz
plt.show()

# Normalización de las variables
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_scaled = pd.DataFrame(X_scaled, columns=X.columns)

# Seleccionar las variables clave
columnas_numericas = ['SalePrice', 'OverallQual', 'GrLivArea', 'TotalBsmtSF', 'GarageCars', 
                      'GarageArea', '1stFlrSF', 'YearBuilt', 'YearRemodAdd', 'FullBath', 
                      'TotRmsAbvGrd']


datos_numericos2 = datost[columnas_numericas]  # Crear nuevo dataframe con las variables seleccionadas

# Mostrar las primeras filas para verificar
columnas_con_nulos = datos_numericos2.columns[datos_numericos2.isnull().any()]

# Rellenar los valores nulos con la mediana de cada columna
datos_numericos2[columnas_con_nulos] = datos_numericos2[columnas_con_nulos].fillna(datos_numericos2[columnas_con_nulos].median())
numericast[columnas_con_nulos] =datos_numericos2[columnas_con_nulos].fillna(datos_numericos2[columnas_con_nulos].median())
# Verificar que no haya valores nulos
print(datos_numericos2.isnull().sum())
print(datos_numericos2.head())

SalePrice       0
OverallQual     0
GrLivArea       0
TotalBsmtSF     0
GarageCars      0
GarageArea      0
1stFlrSF        0
YearBuilt       0
YearRemodAdd    0
FullBath        0
TotRmsAbvGrd    0
dtype: int64
   SalePrice  OverallQual  GrLivArea  TotalBsmtSF  GarageCars  GarageArea  \
0     208500            7       1710          856           2         548   
1     181500            6       1262         1262           2         460   
2     223500            7       1786          920           2         608   
3     140000            7       1717          756           3         642   
4     250000            8       2198         1145           3         836   

   1stFlrSF  YearBuilt  YearRemodAdd  FullBath  TotRmsAbvGrd  
0       856       2003          2003         2             8  
1      1262       1976          1976         2             6  
2       920       2001          2002         2             6  
3       961       1915          1970         1             7  
4      1145       2000          2000         2             9

sns.boxplot(x=datost['OverallQual'], y=datost['SalePrice'])
plt.title('Relación entre la Calidad General de la Vivienda y el Precio')
plt.show()

y = datos_numericos2.pop('SalePrice') #La variable respuesta
X = datos_numericos2 #El resto de los datos
X_train, X_test,y_train, y_test = train_test_split(X, y,test_size=0.3,train_size=0.7)

print(datos_numericos2.isnull().sum())

OverallQual     0
GrLivArea       0
TotalBsmtSF     0
GarageCars      0
GarageArea      0
1stFlrSF        0
YearBuilt       0
YearRemodAdd    0
FullBath        0
TotRmsAbvGrd    0
dtype: int64

#Modelo SalePrice en funcion de OverallQual

saleprice1 = y_train.values.reshape(-1,1)
saleprice1_t = y_test.values.reshape(-1,1)
overallq = X_train['OverallQual'].values.reshape(-1,1)
overallq_t = X_test['OverallQual'].values.reshape(-1,1)
lm1 = LinearRegression()
lm1.fit(overallq, saleprice1)
saleprice1_pred = lm1.predict(overallq_t)

m1 = lm1.coef_[0][0]
c1 = lm1.intercept_[0]


label1 = r'SalePrice = %0.4f*OverallQual % + 0.4f '%(m1,c1)
print(label1)

SalePrice = 45593.7078*OverallQual -98219.5311

#Modelo SalePrice en funcion de GrLivArea

saleprice2 = y_train.values.reshape(-1,1)
saleprice2_t = y_test.values.reshape(-1,1)
gla = X_train['GrLivArea'].values.reshape(-1,1)
gla_t = X_test['GrLivArea'].values.reshape(-1,1)
lm2 = LinearRegression()
lm2.fit(gla, saleprice2)
saleprice2_pred = lm2.predict(gla_t)

m2 = lm2.coef_[0][0]
c2 = lm2.intercept_[0]


label2 = r'SalePrice = %0.4f*GrLivAreax % + 0.4f '%(m2,c2)
print(label2)

SalePrice = 104.7377*GrLivAreax +22804.0545

#Graficos de test

fig = plt.figure()
plt.scatter(overallq_t,saleprice1_t)
plt.plot(overallq_t, saleprice1_pred, color="blue")
plt.xlabel("OverallQual")
plt.ylabel("SalePrice")
plt.title("Test Set OverallQual vs SalePrice")

print("Mean Squared Error: %.2f"%mean_squared_error(saleprice1_t,saleprice1_pred))
print("R squared: %.2f"%r2_score(saleprice1_t,saleprice1_pred))

fig = plt.figure()
plt.scatter(gla_t,saleprice2_t)
plt.plot(gla_t, saleprice2_pred, color="blue")
plt.xlabel("GrLivArea")
plt.ylabel("SalePrice")
plt.title("Test Set GrLivArea vs SalePrice")

print("Mean Squared Error: %.2f"%mean_squared_error(saleprice2_t,saleprice2_pred))
print("R squared: %.2f"%r2_score(saleprice2_t,saleprice2_pred))
print('A pesar de ser las 2 mejores variables en los graficos de dispersion estas no describen idealmente a la variable SalePrice describiendo no mas de un 62 por ciento de su variabilidad.')

Mean Squared Error: 2072674023.49
R squared: 0.66
Mean Squared Error: 2816744119.60
R squared: 0.54
A pesar de ser las 2 mejores variables en los graficos de dispersion estas no describen idealmente a la variable SalePrice describiendo no mas de un 62 por ciento de su variabilidad.

#Graficos de residuales OverallQual

residuales1 = saleprice1_t - saleprice1_pred
plt.plot(overallq_t,residuales1, 'o', color='darkblue')
plt.axhline(0,color='blue')
plt.title("Gráfico de Residuales OverallQual")
plt.xlabel("Variable independiente")
plt.ylabel("Residuales")

print('Los residuos de la variable OverallQual no parecen estar uniformemente distribuidos ademas dado el valor p del test de normalidad tambien podemos descartar la hipotesis nula de que estos sigan una distribucion normal.')

Los residuos de la variable OverallQual no parecen estar uniformemente distribuidos ademas dado el valor p del test de normalidad tambien podemos descartar la hipotesis nula de que estos sigan una distribucion normal.

#Histograma residuales OverallQual

sns.histplot(residuales1, kde=True)
plt.xlabel("Residuales")
plt.title("Distribución de los Residuales")
plt.show()

#Caja y bigote
plt.boxplot(residuales1)

normaltest(residuales1)

NormaltestResult(statistic=array([58.71141458]), pvalue=array([1.78228975e-13]))

#Graficos de residuales GrLivArea

residuales2 = saleprice2_t - saleprice2_pred
plt.plot(gla_t,residuales2, 'o', color='darkblue')
plt.axhline(0,color='blue')
plt.title("Gráfico de Residuales GrLivArea")
plt.xlabel("Variable independiente")
plt.ylabel("Residuales")

print('Los residuos de la variable OverallQual parecen tener cierta tendencia por lo que no estan uniformemente distribuidos ademas dado el valor p del test de normalidad tambien podemos descartar la hipotesis nula de que estos sigan una distribucion normal.')

Los residuos de la variable OverallQual parecen tener cierta tendencia por lo que no estan uniformemente distribuidos ademas dado el valor p del test de normalidad tambien podemos descartar la hipotesis nula de que estos sigan una distribucion normal.

#Histograma de residuales GrLivArea
sns.histplot(residuales2, kde=True)
plt.xlabel("Residuales")
plt.title("Distribución de los Residuales")
plt.show()

#Caja y bigote
plt.boxplot(residuales2)

normaltest(residuales2)

NormaltestResult(statistic=array([68.02229538]), pvalue=array([1.69490841e-15]))

# Analisis de coeficientes OverallQual

est = sm.OLS(saleprice1,overallq)
est2 = est.fit()
print(est2.summary())
print('Dado el valor p podemos descartar la hipotesis nula de que los coeficientes de la variable independiente para el modelo con OverallQual son 0.')

                                 OLS Regression Results                                
=======================================================================================
Dep. Variable:                      y   R-squared (uncentered):                   0.926
Model:                            OLS   Adj. R-squared (uncentered):              0.926
Method:                 Least Squares   F-statistic:                          1.269e+04
Date:                Sun, 02 Mar 2025   Prob (F-statistic):                        0.00
Time:                        23:42:41   Log-Likelihood:                         -12580.
No. Observations:                1021   AIC:                                  2.516e+04
Df Residuals:                    1020   BIC:                                  2.517e+04
Df Model:                           1                                                  
Covariance Type:            nonrobust                                                  
==============================================================================
                 coef    std err          t      P>|t|      [0.025      0.975]
------------------------------------------------------------------------------
x1          3.038e+04    269.738    112.633      0.000    2.99e+04    3.09e+04
==============================================================================
Omnibus:                      662.956   Durbin-Watson:                   1.931
Prob(Omnibus):                  0.000   Jarque-Bera (JB):            10594.371
Skew:                           2.733   Prob(JB):                         0.00
Kurtosis:                      17.804   Cond. No.                         1.00
==============================================================================

Notes:
[1] R² is computed without centering (uncentered) since the model does not contain a constant.
[2] Standard Errors assume that the covariance matrix of the errors is correctly specified.
Dado el valor p podemos descartar la hipotesis nula de que los coeficientes de la variable independiente para el modelo con OverallQual son 0.

# Analisis de coeficiente de GrLivArea

est = sm.OLS(saleprice2,gla)
est2 = est.fit()
print(est2.summary())
print('Dado el valor p podemos descartar la hipotesis nula de que los coeficientes de la variable independiente para el modelo con GrLivArea son 0.')

                                 OLS Regression Results                                
=======================================================================================
Dep. Variable:                      y   R-squared (uncentered):                   0.916
Model:                            OLS   Adj. R-squared (uncentered):              0.916
Method:                 Least Squares   F-statistic:                          1.107e+04
Date:                Sun, 02 Mar 2025   Prob (F-statistic):                        0.00
Time:                        23:42:41   Log-Likelihood:                         -12644.
No. Observations:                1021   AIC:                                  2.529e+04
Df Residuals:                    1020   BIC:                                  2.529e+04
Df Model:                           1                                                  
Covariance Type:            nonrobust                                                  
==============================================================================
                 coef    std err          t      P>|t|      [0.025      0.975]
------------------------------------------------------------------------------
x1           118.0821      1.122    105.205      0.000     115.880     120.285
==============================================================================
Omnibus:                      202.679   Durbin-Watson:                   2.018
Prob(Omnibus):                  0.000   Jarque-Bera (JB):             4545.111
Skew:                          -0.235   Prob(JB):                         0.00
Kurtosis:                      13.326   Cond. No.                         1.00
==============================================================================

Notes:
[1] R² is computed without centering (uncentered) since the model does not contain a constant.
[2] Standard Errors assume that the covariance matrix of the errors is correctly specified.
Dado el valor p podemos descartar la hipotesis nula de que los coeficientes de la variable independiente para el modelo con GrLivArea son 0.

# Crossvalidation

model = LinearRegression()
scores = cross_val_score(model, X_test, y_test, cv=5)  # 5-fold cross-validation
print("Cross-validation scores:", scores)
print("Mean score:", scores.mean())

print('Los modelos planteados parecen estar overfitted por lo que ninguno de los 2 modelos generados parece ser el mejor predictor para la variable SalePrice')

Cross-validation scores: [0.73722476 0.83379408 0.85561901 0.83555305 0.84369257]
Mean score: 0.8211766956414104
Los modelos planteados parecen estar overfitted por lo que ninguno de los 2 modelos generados parece ser el mejor predictor para la variable SalePrice

modelo1 = LinearRegression()
modelo1.fit(X_train, y_train)
y_pred_modelo1 = modelo1.predict(X_test)

#Resumen del modelo
sm.OLS(y_train,X_train).fit().summary()

# calculate aic for regression 
def calculate_aic(n, mse, num_params):
 aic = n * np.log(mse) + 2 * num_params
 return aic
# calculate bic for regression
def calculate_bic(n, mse, num_params):
 bic = n * np.log(mse) + num_params * np.log(n)
 return bic

# Regression metrics
explained_variance_modelo1=metrics.explained_variance_score(y_test, y_pred_modelo1)
mean_absolute_error_modelo1=metrics.mean_absolute_error(y_test, y_pred_modelo1) 
mse_modelo1=metrics.mean_squared_error(y_test, y_pred_modelo1) 
#mean_squared_log_error_modelo1=metrics.mean_squared_log_error(y_test, y_pred_modelo1)
median_absolute_error_modelo1=metrics.median_absolute_error(y_test, y_pred_modelo1)
r2_modelo1=metrics.r2_score(y_test, y_pred_modelo1)
k = modelo1.coef_.size
n = X_train.shape[0]
aic_modelo1 = calculate_aic(n,mse_modelo1,k)
bic_modelo1 = calculate_bic(n,mse_modelo1,k)

print('explained_variance: ', round(explained_variance_modelo1,4))   
#print('mean_squared_log_error: ', round(mean_squared_log_error_modelo1,4))
print('r2: ', round(r2_modelo1,4))
print('MAE: ', round(mean_absolute_error_modelo1,4))
print('MSE: ', round(mse_modelo1,4))
print('RMSE: ', round(np.sqrt(mse_modelo1),4))
print('AIC: ',round(aic_modelo1,4))
print('BIC: ',round(bic_modelo1,4))

explained_variance:  0.8315
r2:  0.831
MAE:  23068.3799
MSE:  1024352062.1458
RMSE:  32005.5005
AIC:  21203.02
BIC:  21252.3053

#Grafico de residuales

resid_modelo1=y_test-y_pred_modelo1
plt.scatter(y_pred_modelo1,resid_modelo1)
plt.axhline(0,color='blue')
plt.xlabel('Valores Predictivos')
plt.ylabel('Residuales')

Text(0, 0.5, 'Residuales')

k,p=sm.stats.diagnostic.lilliefors(resid_modelo1)
print("p =",p)

p = 0.0009999999999998899

x_pred_modelo1 = modelo1.predict(X_train)
mse_modelo1_train=metrics.mean_squared_error(y_train, x_pred_modelo1) 

print("MES_train: ",round(mse_modelo1_train,4))
print('MSE_test: ', round(mse_modelo1,4))

MES_train:  1607545127.32
MSE_test:  1024352062.1458

lr = LinearRegression()

# Crea un modelo de seleccion de predicrores stepwise backward, con un k-fold de 5 en la validación cruzada
seleccionPredictores = SequentialFeatureSelector(lr, direction='backward', cv=5)
seleccionPredictores.fit(X_train, y_train)

# Extrae los índices de los predictores
indicePredictores= seleccionPredictores.get_support(indices=True)



# Variables seleccionadas
columnas = X_train.columns

print("Mejores Predictores para SalePrice:   ")

for i in indicePredictores:
    print(columnas[i])

Mejores Predictores para SalePrice:   
OverallQual
GrLivArea
GarageCars
1stFlrSF
YearBuilt

X_train_stw= X_train.iloc[:,indicePredictores]
X_test_stw = X_test.iloc[:,indicePredictores]

# Train a linear regression model on the training set with the selected features
modelo2 = LinearRegression()
modelo2.fit(X_train_stw, y_train)

# Predict the target values on the testing set
y_pred_modelo2 = modelo2.predict(X_test_stw)

# Regression metrics
explained_variance_modelo2=metrics.explained_variance_score(y_test, y_pred_modelo2)
mean_absolute_error_modelo2=metrics.mean_absolute_error(y_test, y_pred_modelo2) 
mse_modelo2=metrics.mean_squared_error(y_test, y_pred_modelo2) 
#mean_squared_log_error_modelo2=metrics.mean_squared_log_error(y_test, y_pred_modelo2)
median_absolute_error_modelo2=metrics.median_absolute_error(y_test, y_pred_modelo2)
r2_modelo2=metrics.r2_score(y_test, y_pred_modelo2)
k = modelo2.coef_.size
n = X_train.shape[0]
aic_modelo2 = calculate_aic(n,mse_modelo2,k)
bic_modelo2 = calculate_bic(n,mse_modelo2,k)

print('explained_variance: ', round(explained_variance_modelo2,4))   
#print('mean_squared_log_error: ', round(mean_squared_log_error_modelo2,4))
print('r2: ', round(r2_modelo2,4))
print('MAE: ', round(mean_absolute_error_modelo2,4))
print('MSE: ', round(mse_modelo2,4))
print('RMSE: ', round(np.sqrt(mse_modelo2),4))
print('AIC: ',round(aic_modelo2,4))
print('BIC: ',round(bic_modelo2,4))

explained_variance:  0.8177
r2:  0.8172
MAE:  24390.4405
MSE:  1107731852.4937
RMSE:  33282.6059
AIC:  21272.9176
BIC:  21297.5603

resid_modelo2=y_test-y_pred_modelo2
plt.scatter(y_pred_modelo2,resid_modelo2)
plt.axhline(0,color='blue')
plt.xlabel('Valores Predictivos')
plt.ylabel('Residuales')

Text(0, 0.5, 'Residuales')

sm.qqplot(resid_modelo2,line='45')

k,p=sm.stats.diagnostic.lilliefors(resid_modelo2)
print("p =",p)

p = 0.00484558298575972

x_pred_modelo2 = modelo2.predict(X_train_stw)
mse_modelo2_train=metrics.mean_squared_error(y_train, x_pred_modelo2) 

print("MES_train: ",round(mse_modelo2_train,4))
print('MSE_test: ', round(mse_modelo2,4))

MES_train:  1643546634.8412
MSE_test:  1107731852.4937

alphas = np.logspace(-2, 0, num=20)
ridge = make_pipeline(StandardScaler(),
                      RidgeCV(alphas=alphas, store_cv_values=True))


cv = ShuffleSplit(n_splits=5, random_state=1)
cv_results = cross_validate(ridge, X_train, y_train,
                            cv=cv, scoring="neg_mean_squared_error",
                            return_train_score=True,
                            return_estimator=True, n_jobs=2)
train_error = -cv_results["train_score"]
print(f"Error medio cuadrado de la regresión con datos de entrenamiento:\n"
      f"{train_error.mean():.3f} ± {train_error.std():.3f}")

test_error = -cv_results["test_score"]
print(f"Error medio cuadrado de la regresion con los datos de prueba:\n"
      f"{test_error.mean():.3f} ± {test_error.std():.3f}")
mse_alphas = [est[-1].cv_values_.mean(axis=0)
            for est in cv_results["estimator"]]
cv_alphas = pd.DataFrame(mse_alphas, columns=alphas)
cv_alphas

Error medio cuadrado de la regresión con datos de entrenamiento:
1625903528.363 ± 79653727.699
Error medio cuadrado de la regresion con los datos de prueba:
1478052563.156 ± 735320828.210

c:\Users\sebas\AppData\Local\Programs\Python\Python39\lib\site-packages\sklearn\utils\deprecation.py:110: FutureWarning: Attribute `cv_values_` is deprecated in version 1.5 and will be removed in 1.7. Use `cv_results_` instead.
  warnings.warn(msg, category=FutureWarning)

cv_alphas.mean(axis=0).plot(marker="+")
plt.ylabel("Error medio cuadrado\n (menos es mejor)")
plt.xlabel("alpha")
_ = plt.title("Error obtenido en la validación cruzada")

mejores_alphas = [est[-1].alpha_ for est in cv_results["estimator"]]
mejores_alphas

[1.0, 1.0, 1.0, 1.0, 1.0]

print(f"El mejor alfa es:\n"
      f"{np.mean(mejores_alphas):.2f} ± {np.std(mejores_alphas):.2f}")

El mejor alfa es:
1.00 ± 0.00

alfa_modelo3 = np.mean(mejores_alphas)
modelo3 = Ridge(alpha=alfa_modelo3)
modelo3.fit(X_train, y_train)

# Predict the target values on the testing set
y_pred_modelo3 = modelo3.predict(X_test)

# Regression metrics
explained_variance_modelo3=metrics.explained_variance_score(y_test, y_pred_modelo3)
mean_absolute_error_modelo3=metrics.mean_absolute_error(y_test, y_pred_modelo3) 
mse_modelo3=metrics.mean_squared_error(y_test, y_pred_modelo3) 
#mean_squared_log_error_modelo3=metrics.mean_squared_log_error(y_test, y_pred_modelo3)
median_absolute_error_modelo3=metrics.median_absolute_error(y_test, y_pred_modelo3)
r2_modelo3=metrics.r2_score(y_test, y_pred_modelo3)
k = modelo3.coef_.size
n = X_train.shape[0]
aic_modelo3 = calculate_aic(n,mse_modelo3,k)
bic_modelo3 = calculate_bic(n,mse_modelo3,k)

print('explained_variance: ', round(explained_variance_modelo3,4))   
#print('mean_squared_log_error: ', round(mean_squared_log_error_modelo3,4))
print('r2: ', round(r2_modelo3,4))
print('MAE: ', round(mean_absolute_error_modelo3,4))
print('MSE: ', round(mse_modelo3,4))
print('RMSE: ', round(np.sqrt(mse_modelo3),4))
print('AIC: ',round(aic_modelo3,4))
print('BIC: ',round(bic_modelo3,4))

explained_variance:  0.8316
r2:  0.831
MAE:  23061.1141
MSE:  1024214810.8087
RMSE:  32003.3562
AIC:  21202.8832
BIC:  21252.1685

lasso = make_pipeline(StandardScaler(), LassoCV(alphas = np.arange(0,1,0.02), cv=cv, n_jobs = 2))
cv_results_lasso = cross_validate(lasso, X_train, y_train,
                            cv=cv, scoring="neg_mean_squared_error",
                            return_train_score=True,
                            return_estimator=True, n_jobs=2)
train_error = -cv_results_lasso["train_score"]
print(f"Error medio cuadrado de la regresión con datos de entrenamiento:\n"
      f"{train_error.mean():.3f} ± {train_error.std():.3f}")

test_error_lasso = -cv_results_lasso["test_score"]
print(f"Error medio cuadrado de la regresion con los datos de prueba:\n"
      f"{test_error.mean():.3f} ± {test_error.std():.3f}")
alphas = []
for est in cv_results_lasso["estimator"]:
    alphas.append(est[1].alpha_)
print
print(alphas)
-cv_results_lasso["test_score"]

Error medio cuadrado de la regresión con datos de entrenamiento:
1625899407.903 ± 79653465.908
Error medio cuadrado de la regresion con los datos de prueba:
1478052563.156 ± 735320828.210
[0.9400000000000001, 0.9400000000000001, 0.98, 0.98, 0.98]

array([1.17215772e+09, 8.22319876e+08, 2.38600766e+09, 6.81965493e+08,
       2.32619679e+09])

modelo4 = Lasso(alpha=0.04)
modelo4.fit(X_train, y_train)

# Predict the target values on the testing set
y_pred_modelo4 = modelo4.predict(X_test)

# Regression metrics
explained_variance_modelo4=metrics.explained_variance_score(y_test, y_pred_modelo4)
mean_absolute_error_modelo4=metrics.mean_absolute_error(y_test, y_pred_modelo4) 
mse_modelo4=metrics.mean_squared_error(y_test, y_pred_modelo4) 
#mean_squared_log_error_modelo4=metrics.mean_squared_log_error(y_test, y_pred_modelo4)
median_absolute_error_modelo4=metrics.median_absolute_error(y_test, y_pred_modelo4)
r2_modelo4=metrics.r2_score(y_test, y_pred_modelo4)
k = modelo4.coef_.size
n = X_train.shape[0]
aic_modelo4 = calculate_aic(n,mse_modelo4,k)
bic_modelo4 = calculate_bic(n,mse_modelo4,k)

print('explained_variance: ', round(explained_variance_modelo4,4))   
#print('mean_squared_log_error: ', round(mean_squared_log_error_modelo4,4))
print('r2: ', round(r2_modelo4,4))
print('MAE: ', round(mean_absolute_error_modelo4,4))
print('MSE: ', round(mse_modelo4,4))
print('RMSE: ', round(np.sqrt(mse_modelo4),4))
print('AIC: ',round(aic_modelo4,4))
print('BIC: ',round(bic_modelo4,4))

explained_variance:  0.8315
r2:  0.831
MAE:  23068.3614
MSE:  1024351812.7943
RMSE:  32005.4966
AIC:  21203.0197
BIC:  21252.3051

	Id	MSSubClass	LotFrontage	LotArea	OverallQual	OverallCond	YearBuilt	YearRemodAdd	MasVnrArea	BsmtFinSF1	...	WoodDeckSF	OpenPorchSF	EnclosedPorch	3SsnPorch	ScreenPorch	PoolArea	MiscVal	MoSold	YrSold	SalePrice
count	1460.000000	1460.000000	1201.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1452.000000	1460.000000	...	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000	1460.000000
mean	730.500000	56.897260	70.049958	10516.828082	6.099315	5.575342	1971.267808	1984.865753	103.685262	443.639726	...	94.244521	46.660274	21.954110	3.409589	15.060959	2.758904	43.489041	6.321918	2007.815753	180921.195890
std	421.610009	42.300571	24.284752	9981.264932	1.382997	1.112799	30.202904	20.645407	181.066207	456.098091	...	125.338794	66.256028	61.119149	29.317331	55.757415	40.177307	496.123024	2.703626	1.328095	79442.502883
min	1.000000	20.000000	21.000000	1300.000000	1.000000	1.000000	1872.000000	1950.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	2006.000000	34900.000000
25%	365.750000	20.000000	59.000000	7553.500000	5.000000	5.000000	1954.000000	1967.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	5.000000	2007.000000	129975.000000
50%	730.500000	50.000000	69.000000	9478.500000	6.000000	5.000000	1973.000000	1994.000000	0.000000	383.500000	...	0.000000	25.000000	0.000000	0.000000	0.000000	0.000000	0.000000	6.000000	2008.000000	163000.000000
75%	1095.250000	70.000000	80.000000	11601.500000	7.000000	6.000000	2000.000000	2004.000000	166.000000	712.250000	...	168.000000	68.000000	0.000000	0.000000	0.000000	0.000000	0.000000	8.000000	2009.000000	214000.000000
max	1460.000000	190.000000	313.000000	215245.000000	10.000000	9.000000	2010.000000	2010.000000	1600.000000	5644.000000	...	857.000000	547.000000	552.000000	508.000000	480.000000	738.000000	15500.000000	12.000000	2010.000000	755000.000000

	Id	MSSubClass	MSZoning	LotFrontage	LotArea	Street	Alley	LotShape	LandContour	Utilities	...	PoolQC	Fence	MiscFeature	MoSold	YrSold	SaleType	SaleCondition	SalePrice
0	1	60	RL	65.0	8450	Pave	NaN	Reg	Lvl	AllPub	...	NaN	NaN	NaN	2	2008	WD	Normal	208500
1	2	20	RL	80.0	9600	Pave	NaN	Reg	Lvl	AllPub	...	NaN	NaN	NaN	5	2007	WD	Normal	181500
2	3	60	RL	68.0	11250	Pave	NaN	IR1	Lvl	AllPub	...	NaN	NaN	NaN	9	2008	WD	Normal	223500
3	4	70	RL	60.0	9550	Pave	NaN	IR1	Lvl	AllPub	...	NaN	NaN	NaN	2	2006	WD	Abnorml	140000
4	5	60	RL	84.0	14260	Pave	NaN	IR1	Lvl	AllPub	...	NaN	NaN	NaN	12	2008	WD	Normal	250000

	coef	std err	t	P>\|t\|	[0.025	0.975]
OverallQual	2.415e+04	1433.703	16.847	0.000	2.13e+04	2.7e+04
GrLivArea	43.0146	5.350	8.041	0.000	32.517	53.512
TotalBsmtSF	20.5127	5.746	3.570	0.000	9.237	31.788
GarageCars	1.52e+04	4076.321	3.728	0.000	7196.299	2.32e+04
GarageArea	14.5529	13.730	1.060	0.289	-12.390	41.496
1stFlrSF	10.2403	6.692	1.530	0.126	-2.891	23.371
YearBuilt	59.4972	59.372	1.002	0.317	-57.010	176.005
YearRemodAdd	-109.3743	58.418	-1.872	0.061	-224.009	5.261
FullBath	2039.5070	3316.903	0.615	0.539	-4469.297	8548.311
TotRmsAbvGrd	-434.0590	1430.240	-0.303	0.762	-3240.637	2372.519

	0.010000	0.012743	0.016238	0.020691	0.026367	0.033598	0.042813	0.054556	0.069519	0.088587	0.112884	0.143845	0.183298	0.233572	0.297635	0.379269	0.483293	0.615848	0.784760	1.000000
0	1.921462e+09	1.921457e+09	1.921452e+09	1.921444e+09	1.921435e+09	1.921423e+09	1.921408e+09	1.921389e+09	1.921364e+09	1.921333e+09	1.921293e+09	1.921243e+09	1.921178e+09	1.921096e+09	1.920992e+09	1.920859e+09	1.920690e+09	1.920475e+09	1.920202e+09	1.919857e+09
1	1.952919e+09	1.952915e+09	1.952910e+09	1.952903e+09	1.952894e+09	1.952883e+09	1.952868e+09	1.952850e+09	1.952827e+09	1.952797e+09	1.952760e+09	1.952712e+09	1.952651e+09	1.952573e+09	1.952474e+09	1.952348e+09	1.952188e+09	1.951985e+09	1.951727e+09	1.951400e+09
2	1.770770e+09	1.770766e+09	1.770761e+09	1.770754e+09	1.770746e+09	1.770736e+09	1.770722e+09	1.770705e+09	1.770683e+09	1.770656e+09	1.770620e+09	1.770575e+09	1.770518e+09	1.770445e+09	1.770352e+09	1.770234e+09	1.770084e+09	1.769893e+09	1.769651e+09	1.769344e+09
3	1.972672e+09	1.972668e+09	1.972662e+09	1.972655e+09	1.972646e+09	1.972634e+09	1.972619e+09	1.972601e+09	1.972577e+09	1.972546e+09	1.972507e+09	1.972457e+09	1.972394e+09	1.972314e+09	1.972211e+09	1.972081e+09	1.971915e+09	1.971705e+09	1.971438e+09	1.971099e+09
4	1.762940e+09	1.762936e+09	1.762931e+09	1.762925e+09	1.762917e+09	1.762906e+09	1.762893e+09	1.762877e+09	1.762855e+09	1.762828e+09	1.762794e+09	1.762750e+09	1.762694e+09	1.762623e+09	1.762532e+09	1.762417e+09	1.762271e+09	1.762085e+09	1.761849e+09	1.761549e+09

Proyecto 2 Entrega 1¶

Sebastian Garcia 22291¶

Jose Morales 22689¶

Leer datos de un csv¶

- Analisis Exploratorio¶

Resumen del set de datos¶

Descripción de Columnas¶

Obtenemos las columnas con datos Numericos¶

Obtenemos las columnas numericas con datos faltantes¶

Como de bien describen las variables numericas al precio de venta?¶

- Modelos lineales con las variables 'OverallQual' y 'GrLivArea'¶

- Modelo con todas las variables numericas¶

Analisis mejores variables para generar un nuevo modelo¶

Ridge¶

Lasso¶

Dep. Variable:	SalePrice	R-squared (uncentered):	0.958
Model:	OLS	Adj. R-squared (uncentered):	0.957
Method:	Least Squares	F-statistic:	2297.
Date:	Sun, 02 Mar 2025	Prob (F-statistic):	0.00
Time:	23:42:41	Log-Likelihood:	-12290.
No. Observations:	1021	AIC:	2.460e+04
Df Residuals:	1011	BIC:	2.465e+04
Df Model:	10
Covariance Type:	nonrobust

Omnibus:	292.858	Durbin-Watson:	1.985
Prob(Omnibus):	0.000	Jarque-Bera (JB):	27214.374
Skew:	-0.088	Prob(JB):	0.00
Kurtosis:	28.292	Cond. No.	1.18e+04